/* shared_info page starts its life empty. */
shared_info = map_pfn_writeable(pm_handle, shared_info_frame);
memset(shared_info, 0, PAGE_SIZE);
- shared_info->evtchn_upcall_mask = ~0UL; /* mask all upcalls */
+ /* Mask all upcalls... */
+ for ( i = 0; i < MAX_VIRT_CPUS; i++ )
+ shared_info->vcpu_data[i].evtchn_upcall_mask = 1;
unmap_pfn(pm_handle, shared_info);
/* Send the page update requests down to the hypervisor. */
shared_info_t *shared_info;
unsigned long ksize;
mmu_t *mmu = NULL;
- int pm_handle;
+ int pm_handle, i;
memset(builddomain, 0, sizeof(*builddomain));
/* shared_info page starts its life empty. */
shared_info = map_pfn_writeable(pm_handle, shared_info_frame);
memset(shared_info, 0, PAGE_SIZE);
- shared_info->evtchn_upcall_mask = ~0UL; /* mask all upcalls */
+ /* Mask all upcalls... */
+ for ( i = 0; i < MAX_VIRT_CPUS; i++ )
+ shared_info->vcpu_data[i].evtchn_upcall_mask = 1;
unmap_pfn(pm_handle, shared_info);
/* Send the page update requests down to the hypervisor. */
FAILSAFE_ADDR = 36
/* Offsets in shared_info_t */
-UPCALL_PENDING = 0
-UPCALL_MASK = 4
+#define UPCALL_PENDING /* 0 */
+#define UPCALL_MASK 1
/* Offsets in guest_trap_bounce */
GTB_ERROR_CODE = 0
jnz process_hyp_events
/*test_guest_events:*/
movl SHARED_INFO(%ebx),%eax
- movl UPCALL_MASK(%eax),%ecx
- notl %ecx
- andl UPCALL_PENDING(%eax),%ecx # ECX = pending & ~mask
- andl $1,%ecx # Is bit 0 pending and not masked?
+ testb $0xFF,UPCALL_MASK(%eax)
+ jnz restore_all_guest
+ testb $0xFF,UPCALL_PENDING(%eax)
jz restore_all_guest
- lock btsl $0,UPCALL_MASK(%eax) # Upcalls are masked during delivery
+ movb $1,UPCALL_MASK(%eax) # Upcalls are masked during delivery
/*process_guest_events:*/
movzwl PROCESSOR(%ebx),%edx
shl $4,%edx # sizeof(guest_trap_bounce) == 16
gtb->cs = ti->cs;
gtb->eip = ti->address;
if ( TI_GET_IF(ti) )
- set_bit(0, &p->shared_info->evtchn_upcall_mask);
+ p->shared_info->vcpu_data[0].evtchn_upcall_mask = 1;
return;
fault_in_hypervisor:
gtb->cs = ti->cs;
gtb->eip = ti->address;
if ( TI_GET_IF(ti) )
- set_bit(0, &p->shared_info->evtchn_upcall_mask);
- return;
-
+ p->shared_info->vcpu_data[0].evtchn_upcall_mask = 1;
}
asmlinkage void do_double_fault(void)
gtb->cs = ti->cs;
gtb->eip = ti->address;
if ( TI_GET_IF(ti) )
- set_bit(0, &p->shared_info->evtchn_upcall_mask);
+ p->shared_info->vcpu_data[0].evtchn_upcall_mask = 1;
return;
fault_in_hypervisor:
gtb->cs = ti->cs;
gtb->eip = ti->address;
if ( TI_GET_IF(ti) )
- set_bit(0, &p->shared_info->evtchn_upcall_mask);
+ p->shared_info->vcpu_data[0].evtchn_upcall_mask = 1;
return;
gp_in_kernel:
/* Set up shared-info area. */
update_dom_time(p->shared_info);
p->shared_info->domain_time = 0;
- p->shared_info->evtchn_upcall_mask = ~0UL; /* mask all upcalls */
+ /* Mask all upcalls... */
+ for ( i = 0; i < MAX_VIRT_CPUS; i++ )
+ p->shared_info->vcpu_data[i].evtchn_upcall_mask = 1;
/* Install the new page tables. */
__cli();
sched_prn_state(p ->state);
printk(", hyp_events = %08x\n", p->hyp_events);
s = p->shared_info;
- printk("Guest: upcall_pend = %08lx, upcall_mask = %08lx\n",
- s->evtchn_upcall_pending, s->evtchn_upcall_mask);
+ printk("Guest: upcall_pend = %02x, upcall_mask = %02x\n",
+ s->vcpu_data[0].evtchn_upcall_pending,
+ s->vcpu_data[0].evtchn_upcall_mask);
printk("Notifying guest...\n");
send_guest_virq(p, VIRQ_DEBUG);
}
static long do_block(void)
{
ASSERT(current->domain != IDLE_DOMAIN_ID);
- clear_bit(0, ¤t->shared_info->evtchn_upcall_mask);
+ current->shared_info->vcpu_data[0].evtchn_upcall_mask = 0;
current->state = TASK_INTERRUPTIBLE;
TRACE_2D(TRC_SCHED_BLOCK, current->domain, current);
__enter_scheduler();
/* Event channel endpoints per domain. */
#define NR_EVENT_CHANNELS 1024
+/* No support for multi-processor guests. */
+#define MAX_VIRT_CPUS 1
+
/*
* Xen/guestos shared data -- pointer provided in start_info.
* NB. We expect that this struct is smaller than a page.
typedef struct shared_info_st
{
/*
- * If bit 0 in evtchn_upcall_pending is transitioned 0->1, and bit 0 in
- * evtchn_upcall_mask is clear, then an asynchronous upcall is scheduled.
- * The upcall mask can be used to prevent unbounded reentrancy and stack
- * overflow (in this way, acts as a kind of interrupt-enable flag).
+ * Per-VCPU information goes here. This will be cleaned up more when Xen
+ * actually supports multi-VCPU guests.
*/
- unsigned long evtchn_upcall_pending;
- unsigned long evtchn_upcall_mask;
+ struct {
+ /*
+ * 'evtchn_upcall_pending' is written non-zero by Xen to indicate
+ * a pending notification for a particular VCPU. It is then cleared
+ * by the guest OS /before/ checking for pending work, thus avoiding
+ * a set-and-check race. Note that the mask is only accessed by Xen
+ * on the CPU that is currently hosting the VCPU. This means that the
+ * pending and mask flags can be updated by the guest without special
+ * synchronisation (i.e., no need for the x86 LOCK prefix).
+ * This may seem suboptimal because if the pending flag is set by
+ * a different CPU then an IPI may be scheduled even when the mask
+ * is set. However, note:
+ * 1. The task of 'interrupt holdoff' is covered by the per-event-
+ * channel mask bits. A 'noisy' event that is continually being
+ * triggered can be masked at source at this very precise
+ * granularity.
+ * 2. The main purpose of the per-VCPU mask is therefore to restrict
+ * reentrant execution: whether for concurrency control, or to
+ * prevent unbounded stack usage. Whatever the purpose, we expect
+ * that the mask will be asserted only for short periods at a time,
+ * and so the likelihood of a 'spurious' IPI is suitably small.
+ * The mask is read before making an event upcall to the guest: a
+ * non-zero mask therefore guarantees that the VCPU will not receive
+ * an upcall activation. The mask is cleared when the VCPU requests
+ * to block: this avoids wakeup-waiting races.
+ */
+ u8 evtchn_upcall_pending;
+ u8 evtchn_upcall_mask;
+ u8 pad0, pad1;
+ } vcpu_data[MAX_VIRT_CPUS];
/*
* A domain can have up to 1024 "event channels" on which it can send
*/
/* Schedule an asynchronous callback for the specified domain. */
-static inline void __guest_notify(struct task_struct *p)
+static inline void guest_schedule_to_run(struct task_struct *p)
{
#ifdef CONFIG_SMP
unsigned long flags, cpu_mask;
#endif
}
-static inline void guest_notify(struct task_struct *p)
-{
- /*
- * Upcall already pending or upcalls masked?
- * NB. Suitably synchronised on x86:
- * We must set the pending bit before checking the mask, but this is
- * guaranteed to occur because test_and_set_bit() is an ordering barrier.
- */
- if ( !test_and_set_bit(0, &p->shared_info->evtchn_upcall_pending) &&
- !test_bit(0, &p->shared_info->evtchn_upcall_mask) )
- __guest_notify(p);
-}
-
-
/*
* EVENT-CHANNEL NOTIFICATIONS
- * NB. As in guest_notify, evtchn_set_* is suitably synchronised on x86.
+ * NB. On x86, the atomic bit operations also act as memory barriers. There
+ * is therefore sufficiently strict ordering for this architecture -- others
+ * may require explicit memory barriers.
*/
static inline void evtchn_set_pending(struct task_struct *p, int port)
if ( !test_and_set_bit(port, &s->evtchn_pending[0]) &&
!test_bit (port, &s->evtchn_mask[0]) &&
!test_and_set_bit(port>>5, &s->evtchn_pending_sel) )
- guest_notify(p);
+ {
+ /* The VCPU pending flag must be set /after/ update to evtchn-pend. */
+ p->shared_info->vcpu_data[0].evtchn_upcall_pending = 1;
+ guest_schedule_to_run(p);
+ }
}
static inline void evtchn_set_exception(struct task_struct *p, int port)
static inline void send_hyp_event(struct task_struct *p, int event)
{
if ( !test_and_set_bit(event, &p->hyp_events) )
- __guest_notify(p);
+ guest_schedule_to_run(p);
}
/* Called on return from (architecture-dependent) entry.S. */
return 0;
}
-#define signal_pending(_p) \
- (((_p)->hyp_events != 0) || \
- (test_bit(0, &(_p)->shared_info->evtchn_upcall_pending) && \
- !test_bit(0, &(_p)->shared_info->evtchn_upcall_mask)))
+#define signal_pending(_p) \
+ ( (_p)->hyp_events || \
+ ((_p)->shared_info->vcpu_data[0].evtchn_upcall_pending && \
+ !(_p)->shared_info->vcpu_data[0].evtchn_upcall_mask) )
void domain_init(void);
/* Offsets into shared_info_t. */
#define evtchn_upcall_pending /* 0 */
-#define evtchn_upcall_mask 4
+#define evtchn_upcall_mask 1
ENOSYS = 38
movl %eax,EAX(%esp) # save the return value
ENTRY(ret_from_sys_call)
movl SYMBOL_NAME(HYPERVISOR_shared_info),%esi
- lock btsl $0,evtchn_upcall_mask(%esi) # make tests atomic
+ movb $1,evtchn_upcall_mask(%esi) # make tests atomic
ret_syscall_tests:
cmpl $0,need_resched(%ebx)
jne reschedule
cmpl $0,sigpending(%ebx)
je safesti # ensure need_resched updates are seen
signal_return:
- lock btrl $0,evtchn_upcall_mask(%esi) # reenable event callbacks
+ movb $0,evtchn_upcall_mask(%esi) # reenable event callbacks
movl %esp,%eax
xorl %edx,%edx
call SYMBOL_NAME(do_signal)
ALIGN
reschedule:
- lock btrl $0,evtchn_upcall_mask(%esi) # reenable event callbacks
- call SYMBOL_NAME(schedule) # test
- jmp ret_from_sys_call
+ movb $0,evtchn_upcall_mask(%esi) # reenable event callbacks
+ call SYMBOL_NAME(schedule) # test
+ jmp ret_from_sys_call
ENTRY(divide_error)
pushl $0 # no error code
movb CS(%esp),%cl
test $2,%cl # slow return to ring 2 or 3
jne ret_syscall_tests
-safesti:lock btrl $0,evtchn_upcall_mask(%esi) # reenable event callbacks
+safesti:movb $0,evtchn_upcall_mask(%esi) # reenable event callbacks
scrit: /**** START OF CRITICAL REGION ****/
- testb $1,evtchn_upcall_pending(%esi)
+ testb $0xFF,evtchn_upcall_pending(%esi)
jnz 14f # process more events if necessary...
RESTORE_ALL
-14: lock btsl $0,evtchn_upcall_mask(%esi)
+14: movb $1,evtchn_upcall_mask(%esi)
jmp 11b
ecrit: /**** END OF CRITICAL REGION ****/
# [How we do the fixup]. We want to merge the current stack frame with the
jmp 11b
critical_fixup_table:
- .byte 0x00,0x00,0x00 # testb $1,(%esi)
+ .byte 0x00,0x00,0x00 # testb $0xFF,(%esi)
.byte 0x00,0x00 # jnz 14f
.byte 0x00 # pop %ebx
.byte 0x04 # pop %ecx
.byte 0x20 # pop %es
.byte 0x24,0x24,0x24 # add $4,%esp
.byte 0x28 # iret
- .byte 0x00,0x00,0x00,0x00,0x00,0x00 # lock btsl $0,4(%esi)
+ .byte 0x00,0x00,0x00,0x00 # movb $1,4(%esi)
.byte 0x00,0x00 # jmp 11b
# Hypervisor uses this for application faults while it executes.
local_irq_save(flags);
- while ( synch_test_and_clear_bit(0, &s->evtchn_upcall_pending) )
+ while ( s->vcpu_data[0].evtchn_upcall_pending )
{
+ s->vcpu_data[0].evtchn_upcall_pending = 0;
+ /* NB. No need for a barrier here -- XCHG is a barrier on x86. */
l1 = xchg(&s->evtchn_pending_sel, 0);
while ( (l1i = ffs(l1)) != 0 )
{
* a real IO-APIC we 'lose the interrupt edge' if the channel is masked.
*/
if ( synch_test_bit (port, &s->evtchn_pending[0]) &&
- !synch_test_and_set_bit(port>>5, &s->evtchn_pending_sel) &&
- !synch_test_and_set_bit(0, &s->evtchn_upcall_pending) &&
- !synch_test_bit (0, &s->evtchn_upcall_mask) )
- evtchn_do_upcall(NULL);
+ !synch_test_and_set_bit(port>>5, &s->evtchn_pending_sel) )
+ {
+ s->vcpu_data[0].evtchn_upcall_pending = 1;
+ if ( !s->vcpu_data[0].evtchn_upcall_mask )
+ evtchn_do_upcall(NULL);
+ }
}
static inline void clear_evtchn(int port)
#define safe_halt() ((void)0)
-/*
- * Note the use of synch_*_bit() operations in the following. These operations
- * ensure correct serialisation of checks and updates w.r.t. Xen executing on
- * a different CPU.
+/*
+ * The use of 'barrier' in the following reflects their use as local-lock
+ * operations. Reentrancy must be prevented (e.g., __cli()) /before/ following
+ * critical operations are executed. All critical operatiosn must complete
+ * /before/ reentrancy is permitted (e.g., __sti()). Alpha architecture also
+ * includes these barriers, for example.
*/
#define __cli() \
do { \
- synch_set_bit(0, &HYPERVISOR_shared_info->evtchn_upcall_mask); \
+ HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask = 1; \
+ barrier(); \
} while (0)
#define __sti() \
do { \
shared_info_t *_shared = HYPERVISOR_shared_info; \
- synch_clear_bit(0, &_shared->evtchn_upcall_mask); \
- if ( unlikely(synch_test_bit(0, &_shared->evtchn_upcall_pending)) ) \
+ barrier(); \
+ _shared->vcpu_data[0].evtchn_upcall_mask = 0; \
+ if ( unlikely(_shared->vcpu_data[0].evtchn_upcall_pending) ) \
evtchn_do_upcall(NULL); \
} while (0)
#define __save_flags(x) \
do { \
- (x) = synch_test_bit(0, &HYPERVISOR_shared_info->evtchn_upcall_mask); \
+ (x) = HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask; \
} while (0)
-#define __restore_flags(x) do { if (x) __cli(); else __sti(); } while (0)
+#define __restore_flags(x) \
+do { \
+ shared_info_t *_shared = HYPERVISOR_shared_info; \
+ barrier(); \
+ if ( (_shared->vcpu_data[0].evtchn_upcall_mask = x) == 0 ) \
+ if ( unlikely(_shared->vcpu_data[0].evtchn_upcall_pending) ) \
+ evtchn_do_upcall(NULL); \
+} while (0)
#define __save_and_cli(x) \
do { \
- (x) = synch_test_and_set_bit( \
- 0, &HYPERVISOR_shared_info->evtchn_upcall_mask); \
+ (x) = HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask; \
+ HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask = 1; \
+ barrier(); \
} while (0)
#define __save_and_sti(x) \
do { \
- (x) = synch_test_and_clear_bit( \
- 0, &HYPERVISOR_shared_info->evtchn_upcall_mask); \
+ barrier(); \
+ (x) = HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask; \
+ HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask = 0; \
} while (0)
#define local_irq_save(x) __save_and_cli(x)
projects / xen.git / commitdiff